
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import seaborn as sns
import scipy.stats
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
import seaborn as sns
from scipy import stats


data = pd.read_csv('../datasets/AB_NYC_2019.csv')


data.head()


data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 48895 entries, 0 to 48894
Data columns (total 16 columns):
id                                48895 non-null int64
name                              48879 non-null object
host_id                           48895 non-null int64
host_name                         48874 non-null object
neighbourhood_group               48895 non-null object
neighbourhood                     48895 non-null object
latitude                          48895 non-null float64
longitude                         48895 non-null float64
room_type                         48895 non-null object
price                             48895 non-null int64
minimum_nights                    48895 non-null int64
number_of_reviews                 48895 non-null int64
last_review                       38843 non-null object
reviews_per_month                 38843 non-null float64
calculated_host_listings_count    48895 non-null int64
availability_365                  48895 non-null int64
dtypes: float64(3), int64(7), object(6)
memory usage: 6.0+ MB


data.isnull().sum()

id                                    0
name                                 16
host_id                               0
host_name                            21
neighbourhood_group                   0
neighbourhood                         0
latitude                              0
longitude                             0
room_type                             0
price                                 0
minimum_nights                        0
number_of_reviews                     0
last_review                       10052
reviews_per_month                 10052
calculated_host_listings_count        0
availability_365                      0
dtype: int64


print (data['id'].nunique(),len(data))

48895 48895


print (data['host_id'].nunique(),len(data))

37457 48895


host_ids=data.loc[data['host_name'].isnull()]['host_id']
for id  in host_ids:
    display(data.loc[data['host_id'] == id])


plt.figure(figsize=(10, 6))
sns.scatterplot(data.longitude, data.latitude, hue=data.neighbourhood_group)
plt.ioff()


host_gender = data[['host_id','host_name','price','minimum_nights','number_of_reviews','calculated_host_listings_count']]
host_gender= host_gender[pd.notnull(host_gender['host_name'])]
host_gender.head()


import  gender_guesser.detector as gender
d = gender.Detector()


host_gender['gender']=host_gender['host_name'].apply(lambda x : d.get_gender(x))
host_gender.head()


host_gender.groupby('gender').count()


display(host_gender.loc[host_gender['gender'] == 'andy'].head())
display(host_gender.loc[host_gender['gender'] == 'unknown'].head())


host_gender=host_gender.drop(host_gender[host_gender.gender=='andy'].index)
host_gender=host_gender.drop(host_gender[host_gender.gender=='unknown'].index)
host_gender.loc[host_gender['gender'] == 'mostly_female', ['gender']] = 'female'
host_gender.loc[host_gender['gender'] == 'mostly_male', ['gender']] = 'male'
host_gender.loc[host_gender['gender'] == 'male', ['gender']] = 1
host_gender.loc[host_gender['gender'] == 'female', ['gender']] = 0
host_gender=host_gender.drop(['host_name'],axis=1)


corr = host_gender.corr()
corr.style.background_gradient(cmap='coolwarm')


print('---Male')
display(host_gender[host_gender.gender==1].describe())
print('---Female')
display(host_gender[host_gender.gender==0].describe())

---Male

---Female


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].price, host_gender[host_gender.gender==0].price))
ax = sns.violinplot(x="gender", y="price",data=host_gender[host_gender.price < 500],scale="width")

Ttest_indResult(statistic=1.7918969601450074, pvalue=0.07315755802542322)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].number_of_reviews, host_gender[host_gender.gender==0].number_of_reviews))
ax = sns.violinplot(x="gender", y="number_of_reviews",data=host_gender,scale="width")

Ttest_indResult(statistic=2.043856646336926, pvalue=0.04097470461432081)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].calculated_host_listings_count, host_gender[host_gender.gender==0].calculated_host_listings_count))
ax = sns.violinplot(x="gender", y="calculated_host_listings_count",data=host_gender,scale="width")

Ttest_indResult(statistic=5.610587116141991, pvalue=2.030656769150408e-08)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].minimum_nights, host_gender[host_gender.gender==0].minimum_nights))
ax = sns.violinplot(x="gender", y="minimum_nights",data=host_gender[host_gender.minimum_nights<40],scale="width")

Ttest_indResult(statistic=-0.06396568177457355, pvalue=0.9489978948270685)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].number_of_reviews, host_gender[host_gender.gender==0].number_of_reviews))
ax = sns.violinplot(x="gender", y="number_of_reviews", data=host_gender[host_gender.number_of_reviews < 100], scale="width")

Ttest_indResult(statistic=2.043856646336926, pvalue=0.04097470461432081)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].minimum_nights, host_gender[host_gender.gender==0].minimum_nights))
ax = sns.violinplot(x="gender", y="minimum_nights", data=host_gender[host_gender.minimum_nights < 50], scale="width")

Ttest_indResult(statistic=-0.06396568177457355, pvalue=0.9489978948270685)


print(scipy.stats.ttest_ind(host_gender[host_gender.gender==1].calculated_host_listings_count, host_gender[host_gender.gender==0].calculated_host_listings_count))
ax = sns.violinplot(x="gender", y="calculated_host_listings_count",data=host_gender[host_gender.calculated_host_listings_count < 15], scale="width")

Ttest_indResult(statistic=5.610587116141991, pvalue=2.030656769150408e-08)


labels = host_gender.gender.value_counts().index
explode = [0,0]
sizes = host_gender.gender.value_counts().values
plt.pie(sizes, explode=explode, labels=labels,autopct='%1.1f%%')
plt.show()


print (data['neighbourhood'].nunique(),len(data))
print (data['neighbourhood_group'].nunique(),len(data))

221 48895
5 48895


name_neighbourhood=data[['name','neighbourhood','neighbourhood_group','price']]
name_neighbourhood.head()


from sklearn.cluster import MiniBatchKMeans
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.metrics.cluster import adjusted_rand_score


tfidf = TfidfVectorizer(min_df = 500, max_df = 0.95, max_features = 8000, stop_words = 'english')
tfidf.fit(name_neighbourhood['name'].values.astype(str))
text = tfidf.transform(name_neighbourhood['name'].values.astype(str))


clusters = MiniBatchKMeans(n_clusters=5, init_size=1024, batch_size=2048, random_state=20).fit_predict(text)


adjusted_rand_score(clusters,name_neighbourhood.neighbourhood_group)

0.0036952482090660834


def plot_tsne_pca(data, labels):
    max_label = max(labels)
    max_items = np.random.choice(range(data.shape[0]), size=3000, replace=False)
    
    pca = PCA(n_components=2).fit_transform(data[max_items,:].todense())
    tsne = TSNE().fit_transform(PCA(n_components=50).fit_transform(data[max_items,:].todense()))
    
    
    idx = np.random.choice(range(pca.shape[0]), size=300, replace=False)
    label_subset = labels[max_items]
    label_subset = [cm.hsv(i/max_label) for i in label_subset[idx]]
    
    f, ax = plt.subplots(1, 2, figsize=(14, 6))
    
    ax[0].scatter(pca[idx, 0], pca[idx, 1], c=label_subset)
    ax[0].set_title('PCA Cluster Plot')
    
    ax[1].scatter(tsne[idx, 0], tsne[idx, 1], c=label_subset)
    ax[1].set_title('TSNE Cluster Plot')
    
plot_tsne_pca(text, clusters)


def get_top_keywords(data, clusters, labels, n_terms):
    df = pd.DataFrame(data.todense()).groupby(clusters).mean()
    
    for i,r in df.iterrows():
        print('\nCluster {}'.format(i))
        print(','.join([labels[t] for t in np.argsort(r)[-n_terms:]]))
            
get_top_keywords(text, clusters, tfidf.get_feature_names(), 10)

Cluster 0
apt,williamsburg,east,beautiful,sunny,brooklyn,spacious,cozy,apartment,bedroom

Cluster 1
nyc,williamsburg,cozy,park,manhattan,loft,1br,home,brooklyn,apt

Cluster 2
apt,williamsburg,brooklyn,manhattan,bath,bathroom,cozy,bedroom,room,private

Cluster 3
beautiful,near,large,williamsburg,brooklyn,manhattan,sunny,spacious,cozy,room

Cluster 4
west,private,manhattan,spacious,park,village,apartment,east,cozy,studio


clusters = MiniBatchKMeans(n_clusters=221, init_size=1024, batch_size=2048, random_state=20).fit_predict(text)
plot_tsne_pca(text, clusters)


adjusted_rand_score(clusters,name_neighbourhood.neighbourhood)

0.06691539622141146


classic = data[['neighbourhood_group','neighbourhood','latitude','longitude','room_type','price','minimum_nights','number_of_reviews','availability_365']]


classic.head()


fig, axes = plt.subplots(1,2, figsize=(21,6))
sns.distplot(classic[classic.price<800]['price'], ax=axes[0])
sns.distplot(classic[classic.price>800]['price'], ax=axes[1])

<matplotlib.axes._subplots.AxesSubplot at 0x23b82b03908>


fig, axes = plt.subplots(2,1, figsize=(18,9))
sns.violinplot(classic['neighbourhood_group'], classic[classic.price<800]['price'],ax=axes[0])
sns.violinplot(classic['neighbourhood_group'], classic[classic.price>800]['price'],ax=axes[1])
plt.show()


fig, axes = plt.subplots(5,1, figsize=(100,90))
sns.violinplot(classic[classic.neighbourhood_group=='Brooklyn']['neighbourhood'], classic[classic.price<800]['price'],ax=axes[0])
sns.violinplot(classic[classic.neighbourhood_group=='Manhattan']['neighbourhood'], classic[classic.price<800]['price'],ax=axes[1])
sns.violinplot(classic[classic.neighbourhood_group=='Queens']['neighbourhood'], classic[classic.price<800]['price'],ax=axes[2])
sns.violinplot(classic[classic.neighbourhood_group=='Staten Island']['neighbourhood'], classic[classic.price<800]['price'],ax=axes[3])
sns.violinplot(classic[classic.neighbourhood_group=='Bronx']['neighbourhood'], classic[classic.price<800]['price'],ax=axes[4])
plt.show()


fig, axes = plt.subplots(5,1, figsize=(100,90))
sns.violinplot(classic[classic.neighbourhood_group=='Brooklyn']['neighbourhood'], classic[classic.price>800]['price'],ax=axes[0])
sns.violinplot(classic[classic.neighbourhood_group=='Manhattan']['neighbourhood'], classic[classic.price>800]['price'],ax=axes[1])
sns.violinplot(classic[classic.neighbourhood_group=='Queens']['neighbourhood'], classic[classic.price>800]['price'],ax=axes[2])
sns.violinplot(classic[classic.neighbourhood_group=='Staten Island']['neighbourhood'], classic[classic.price>800]['price'],ax=axes[3])
sns.violinplot(classic[classic.neighbourhood_group=='Bronx']['neighbourhood'], classic[classic.price>800]['price'],ax=axes[4])
plt.show()


sns.catplot(x='neighbourhood_group', kind='count' ,data=classic)

<seaborn.axisgrid.FacetGrid at 0x23bf77696a0>


sns.catplot(x='room_type', kind='count' ,data=classic)

<seaborn.axisgrid.FacetGrid at 0x23b815e7518>


fig, axes = plt.subplots(2,1, figsize=(18,9))
sns.violinplot(classic['room_type'], classic[classic.price<800]['price'],ax=axes[0])
sns.violinplot(classic['room_type'], classic[classic.price>800]['price'],ax=axes[1])
plt.show()


sns.violinplot(classic['room_type'], classic[classic.availability_365>0]['availability_365'])

E:\ProgramData\lib\site-packages\scipy\stats\stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval

<matplotlib.axes._subplots.AxesSubplot at 0x23b82324a20>


sns.violinplot(classic['room_type'], classic[classic.minimum_nights<40]['minimum_nights'])

<matplotlib.axes._subplots.AxesSubplot at 0x23b82336ba8>


sns.distplot(classic['minimum_nights'])
fig = plt.gcf()
fig.set_size_inches(10,5)


sns.violinplot(classic['neighbourhood_group'], classic[classic.minimum_nights<40]['minimum_nights'])

<matplotlib.axes._subplots.AxesSubplot at 0x23b85dd6cf8>


sns.distplot(classic[classic.availability_365>0]['availability_365'])
fig = plt.gcf()
fig.set_size_inches(10,5)

E:\ProgramData\lib\site-packages\scipy\stats\stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval


sns.violinplot(classic['neighbourhood_group'], classic['availability_365'])

<matplotlib.axes._subplots.AxesSubplot at 0x23b85821128>


fig, axes = plt.subplots(5,1, figsize=(100,90))
sns.violinplot(classic[classic.neighbourhood_group=='Brooklyn']['neighbourhood'], classic['availability_365'],ax=axes[0])
sns.violinplot(classic[classic.neighbourhood_group=='Manhattan']['neighbourhood'], classic['availability_365'],ax=axes[1])
sns.violinplot(classic[classic.neighbourhood_group=='Queens']['neighbourhood'], classic['availability_365'],ax=axes[2])
sns.violinplot(classic[classic.neighbourhood_group=='Staten Island']['neighbourhood'], classic['availability_365'],ax=axes[3])
sns.violinplot(classic[classic.neighbourhood_group=='Bronx']['neighbourhood'], classic['availability_365'],ax=axes[4])
plt.show()


neighborhoods_for_each_group = {}
for ngh_group in neighbourhood_group:
    neighborhoods_for_each_group[ngh_group] = {}
    neighborhoods_for_each_group[ngh_group]["places"] = data[data.neighbourhood_group == ngh_group].neighbourhood.unique()
    print(ngh_group, ' -> ', neighborhoods_for_each_group[ngh_group]['places'].size)

Brooklyn  ->  47
Manhattan  ->  32
Queens  ->  51
Staten Island  ->  43
Bronx  ->  48


data[data.neighbourhood_group == "Bronx"].number_of_reviews.sum()
for ngh_group in neighborhoods_for_each_group:
    neighborhoods_for_each_group[ngh_group]["number_of_reviews"] = data[data.neighbourhood_group == ngh_group].number_of_reviews.sum()
# sns.barplot(y="neighbourhood_group", x="number_of_reviews", data=data)


viz = []
for ngh in neighborhoods_for_each_group:
    viz.append([ngh, neighborhoods_for_each_group[ngh]['number_of_reviews']])
test = pd.DataFrame(viz, columns=['group', 'total_review'])


plt.figure(figsize=(9, 5))
sns.barplot(x= 'group', y='total_review', data=test)
plt.show()


plt.figure(figsize=(9, 5))
sns.countplot(x="neighbourhood_group", data=data)
plt.show()


plt.figure(figsize=(15, 9))
sns.violinplot(x='neighbourhood_group', y='price', data=data)

<matplotlib.axes._subplots.AxesSubplot at 0x7fde4b6644e0>


plt.figure(figsize=(12, 8))
sns.violinplot(x='neighbourhood_group', y='price', data=data[data['price'] < 1000])

<matplotlib.axes._subplots.AxesSubplot at 0x7fde4b5b2080>


brooklyn_price = data[data['neighbourhood_group'] == 'Brooklyn'].price.values
manhatan_price = data[data['neighbourhood_group'] == 'Manhattan'].price.values
queens_price = data[data['neighbourhood_group'] == 'Queens'].price.values
staten_island_price = data[data['neighbourhood_group'] == 'Staten Island'].price.values
bronx_price = data[data['neighbourhood_group'] == 'Bronx'].price.values


stats.f_oneway(bronx_price, manhatan_price, queens_price, staten_island_price, brooklyn_price)

F_onewayResult(statistic=354.12630761916523, pvalue=4.208501569800712e-301)


plt.figure(figsize=(10, 6))
sns.kdeplot(data['price'])
plt.xlabel('Values')
plt.ylabel('Frequency')
plt.show()
# useful for bining price


price_mean = data['price'].mean()
price_std = data['price'].std()
print('price mean = ', price_mean)
print('price std = ', price_std)
print('price min = ', data['price'].min())
print('price max = ', data['price'].max())

price mean =  152.77470469015498
price std =  240.24849763081855
price min =  10
price max =  10000


# bining price
price_bins = [10, 153, 393, 633, 10000]
labels = [1, 2, 3, 4]
data['price_bin'] = pd.cut(data['price'], bins = price_bins, labels = labels)


plt.figure(figsize=(20, 12))
sns.catplot(x='neighbourhood_group',col='price_bin', kind='count', data=data)
plt.show()

<Figure size 1440x864 with 0 Axes>


sns.catplot(x='price_bin',col='neighbourhood_group', kind='count', data=data)

<seaborn.axisgrid.FacetGrid at 0x7fde4aeec748>


plt.figure(figsize=(10, 6))
sns.kdeplot(data['minimum_nights'])
plt.xlabel('Values')
plt.ylabel('Frequency')
plt.show()


print('minimum_nights mean =', data['minimum_nights'].mean())
print('minimum_nights std =', data['minimum_nights'].std())

minimum_nights mean = 7.012365140131431
minimum_nights std = 20.02136215911567


print('minimum_nights min =', data['minimum_nights'].min())
print('minimum_nights max =', data['minimum_nights'].max())

minimum_nights min = 1
minimum_nights max = 1250


# plt.figure(figsize=(20, 15))
fig, axes = plt.subplots(2,1, figsize=(18,12))
sns.countplot(x='minimum_nights', data=data[data['minimum_nights'] <= 89 ],  ax=axes[0])
sns.countplot(x='minimum_nights', data=data[data['minimum_nights'] > 89 ],  ax=axes[1])
plt.show()


plt.figure(figsize=(20, 15))
fig, axes = plt.subplots(2,1, figsize=(18,12))
data_min_night_100 = data[data['minimum_nights'] <= 100]
sns.stripplot(x='minimum_nights', y='price', data=data_min_night_100[data_min_night_100['price'] <= 1000], ax=axes[0])
sns.stripplot(x='minimum_nights', y='price', data=data_min_night_100[data_min_night_100['price'] > 1000], ax=axes[1])
plt.show()

<Figure size 1440x1080 with 0 Axes>


data[data['minimum_nights'] >= 30].size

67545


sns.jointplot(data=data[data.price<300][data.minimum_nights<20],x="minimum_nights",y='price',kind="kde")
# sns.lmplot(data=data[data.price<600][data.minimum_nights<100],y="minimum_nights",x='price')
plt.show()

/home/alireza/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  """Entry point for launching an IPython kernel.


fig, axes = plt.subplots(2,1, figsize=(18,12))
sns.violinplot(x='price_bin', y="minimum_nights", data=data[data.minimum_nights < 50 ], ax=axes[0])
sns.violinplot(x='price_bin', y="minimum_nights", data=data[data.minimum_nights > 50 ], ax=axes[1])
plt.show()


price_bin_1 = data[data.price_bin == 1].minimum_nights.values
price_bin_2 = data[data.price_bin == 2].minimum_nights.values
price_bin_3 = data[data.price_bin == 3].minimum_nights.values
price_bin_4 = data[data.price_bin == 4].minimum_nights.values


# note: 
#  this is ANOVA test is for all data not just for those minimum_nights  < 40
stats.f_oneway(price_bin_1, price_bin_3, price_bin_4)

F_onewayResult(statistic=15.653877892980217, pvalue=1.6015938901792606e-07)


print("mean")
print("price_bin_1 : ", data[data.price_bin == 1].minimum_nights.mean())
print("price_bin_2 : ", data[data.price_bin == 2].minimum_nights.mean())
print("price_bin_3 : ", data[data.price_bin == 3].minimum_nights.mean())
print("price_bin_4 : ", data[data.price_bin == 4].minimum_nights.mean())

mean
price_bin_1 :  6.569271967512212
price_bin_2 :  8.01447223533113
price_bin_3 :  6.7468910021945865
price_bin_4 :  10.571056062581487


data[data.minimum_nights > 100].size

2595


min_night_30 = data[data.minimum_nights == 30]
min_night_30.head()


sns.countplot(x='price_bin', data=min_night_30)

<matplotlib.axes._subplots.AxesSubplot at 0x7fde4a785f98>


print("mean")
print("price_bin_1 : ", len( min_night_30[min_night_30.price_bin == 1]))
print("price_bin_2 : ", len(min_night_30[min_night_30.price_bin == 2]))
print("price_bin_3 : ", len(min_night_30[min_night_30.price_bin == 3]))
print("price_bin_4 : ", len(min_night_30[min_night_30.price_bin == 4]))

mean
price_bin_1 :  2182
price_bin_2 :  1402
price_bin_3 :  109
price_bin_4 :  63


plt.figure(figsize=(20, 15))
sns.violinplot(x='neighbourhood_group', y='minimum_nights', data=data[data['minimum_nights'] < 40])
plt.show()


Brooklyn_night = data[data.neighbourhood_group == 'Brooklyn'].minimum_nights.values
Monhatan_night = data[data.neighbourhood_group == 'Manhattan'].minimum_nights.values
Queens_night = data[data.neighbourhood_group == 'Queens'].minimum_nights.values
Stateen_Island_night = data[data.neighbourhood_group == 'Staten Island'].minimum_nights.values
Bronx_night = data[data.neighbourhood_group == 'Bronx'].minimum_nights.values


print("mean")
print("Brooklyn_night : ", Brooklyn_night.mean())
print("Monhatan_night : ", Monhatan_night.mean())
print("Queens_night : ", Queens_night.mean())
print("Stateen_Island_night : ", Stateen_Island_night.mean())
print("Bronx_night : ", Bronx_night.mean())

mean
Brooklyn_night :  6.056623505976096
Monhatan_night :  8.538443766749838
Queens_night :  5.182909604519774
Stateen_Island_night :  4.831099195710456
Bronx_night :  4.567095588235294


stats.f_oneway(Brooklyn_night, Monhatan_night, Queens_night, Stateen_Island_night, Bronx_night)

F_onewayResult(statistic=60.150404756022375, pvalue=9.205204002897539e-51)


# trim mean data
print("trim mean")
print("Brooklyn_night : ", stats.trim_mean(Brooklyn_night, 0.1))
print("Monhatan_night : ",stats.trim_mean(Monhatan_night, 0.1))
print("Queens_night : ", stats.trim_mean(Queens_night, 0.1))
print("Stateen_Island_night : ", stats.trim_mean(Stateen_Island_night, 0.1))
print("Bronx_night : ", stats.trim_mean(Bronx_night, 0.1))

trim mean
Brooklyn_night :  3.104581673306773
Monhatan_night :  5.098937276192676
Queens_night :  2.409973521624007
Stateen_Island_night :  2.31438127090301
Bronx_night :  2.161697247706422


plt.figure(figsize=(20, 20))
sns.boxplot(x='room_type', y='minimum_nights', data=data[data.minimum_nights < 200])
plt.show()


private_room_night = data[data.room_type == 'Private room'].minimum_nights.values
entire_room_night = data[data.room_type == 'Entire home/apt'].minimum_nights.values
shared_room_night = data[data.room_type == 'Shared room'].minimum_nights.values


print("mean")
print("private : ", private_room_night.mean())
print("entire : ", entire_room_night.mean())
print("shared : ", shared_room_night.mean())

mean
private :  5.381855688595901
entire :  8.470442282698594
shared :  6.439066551426102


print("mean")
print("private : ", stats.trim_mean(private_room_night, 0.1))
print("entire : ", stats.trim_mean(entire_room_night, 0.1))
print("shared : ", stats.trim_mean(shared_room_night, 0.09))

mean
private :  2.6099433888234964
entire :  5.053906365381775
shared :  2.724973656480506


price_data_frame = data.price.values


from sklearn.cluster import KMeans


price_data_frame=price_data_frame.reshape(-1, 1)
kmeans = KMeans(n_clusters=3, max_iter=300)
kmeans.fit(price_data_frame)

KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,
    n_clusters=3, n_init=10, n_jobs=None, precompute_distances='auto',
    random_state=None, tol=0.0001, verbose=0)


price_labels = kmeans.labels_


kmeans.cluster_centers_

array([[ 132.48881282],
       [5845.86046512],
       [ 967.94843049]])


price_labels

array([0, 0, 0, ..., 0, 0, 0], dtype=int32)


data['label'] = price_labels


cluster_0 = data[data.label == 0]
cluster_1 = data[data.label == 1]
cluster_2 = data[data.label == 2]


cluster_0.head()


test_0 = pd.DataFrame({'price': cluster_0.price, 'label': cluster_0.label})
test_1 = pd.DataFrame({'price': cluster_1.price, 'label': cluster_1.label})
test_2 = pd.DataFrame({'price': cluster_2.price, 'label': cluster_2.label})


test_0.describe().T


test_1.describe().T


test_2.describe().T


plt.figure(figsize=(8, 5))
sns.kdeplot(test_0.price)
plt.xlabel('Price')
plt.ylabel('Frequency')
plt.show()


plt.figure(figsize=(8, 5))
sns.kdeplot(test_1.price)
plt.xlabel('Price')
plt.ylabel('Frequency')
plt.show()


plt.figure(figsize=(8, 5))
sns.kdeplot(test_2.price)
plt.xlabel('Price')
plt.ylabel('Frequency')
plt.show()


print("cluster")
print("mean", cluster_0.price.mean())
print("std", cluster_0.price.std())
print("min", cluster_0.price.min())
print("max", cluster_0.price.max())

cluster
mean 132.48881282350976
std 90.05980054092642
min 10
max 550


# stats.normaltest(cluster_0.price.values)


cluster_0.price.mean()

132.48881282350976


kmeans = KMeans(n_clusters=3, max_iter=300)
kmeans.fit(cluster_0.price.values.reshape(-1,1))

KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,
    n_clusters=3, n_init=10, n_jobs=None, precompute_distances='auto',
    random_state=None, tol=0.0001, verbose=0)


kmeans.cluster_centers_

array([[ 77.78074164],
       [358.25745885],
       [184.57482732]])


cluster_0['low_price_label'] = kmeans.labels_

/home/alireza/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.


# cluster_0[cluster_0.low_price_label == 1].low_price_label = 3
# cluster_0[cluster_0.low_price_label == 2].low_price_label = 1
# cluster_0[cluster_0.low_price_label == 3].low_price_label = 2


cluster_0[cluster_0.low_price_label == 0].price.describe()

count    29691.000000
mean        77.780742
std         27.657951
min         10.000000
25%         55.000000
50%         75.000000
75%        100.000000
max        131.000000
Name: price, dtype: float64


cluster_0[cluster_0.low_price_label == 1].price.describe()

count    3888.000000
mean      358.257459
std        73.379913
min       272.000000
25%       300.000000
50%       349.000000
75%       400.000000
max       550.000000
Name: price, dtype: float64


cluster_0[cluster_0.low_price_label == 2].price.describe()

count    14333.000000
mean       184.574827
std         36.436127
min        132.000000
25%        150.000000
50%        179.000000
75%        200.000000
max        271.000000
Name: price, dtype: float64


plt.figure(figsize=(20, 12))
sns.catplot(x='neighbourhood_group',col='low_price_label', kind='count', data=cluster_0)
plt.show()

<Figure size 1440x864 with 0 Axes>


data.corr()['host_id']

id                                0.588290
host_id                           1.000000
latitude                          0.020224
longitude                         0.127055
price                             0.015309
minimum_nights                   -0.017364
number_of_reviews                -0.140106
reviews_per_month                 0.296417
calculated_host_listings_count    0.154950
availability_365                  0.203492
Name: host_id, dtype: float64

	id	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	last_review	reviews_per_month	calculated_host_listings_count	availability_365
0	2539	Clean & quiet apt home by the park	2787	John	Brooklyn	Kensington	40.64749	-73.97237	Private room	149	1	9	2018-10-19	0.21	6	365
1	2595	Skylit Midtown Castle	2845	Jennifer	Manhattan	Midtown	40.75362	-73.98377	Entire home/apt	225	1	45	2019-05-21	0.38	2	355
2	3647	THE VILLAGE OF HARLEM....NEW YORK !	4632	Elisabeth	Manhattan	Harlem	40.80902	-73.94190	Private room	150	3	0	NaN	NaN	1	365
3	3831	Cozy Entire Floor of Brownstone	4869	LisaRoxanne	Brooklyn	Clinton Hill	40.68514	-73.95976	Entire home/apt	89	1	270	2019-07-05	4.64	1	194
4	5022	Entire Apt: Spacious Studio/Loft by central park	7192	Laura	Manhattan	East Harlem	40.79851	-73.94399	Entire home/apt	80	10	9	2018-11-19	0.10	1	0

	id	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	last_review	reviews_per_month	calculated_host_listings_count
27962	21867710	Amazing room in the centre on Manhatten! Welcome!	159156636	NaN	Manhattan	Hell's Kitchen	40.75668	-73.99097	Private room	120	1	89	2019-01-01	5.16	3
28042	21901156	Best Deal! Lovely place in Manhattan! Time Squ...	159156636	NaN	Manhattan	Hell's Kitchen	40.75656	-73.99063	Private room	120	1	109	2019-01-01	5.97	3
28274	21990808	*SLEEPING IN THE GOLDEN PLACE*	159156636	NaN	Manhattan	Hell's Kitchen	40.75835	-73.99065	Private room	120	1	88	2018-12-14	4.93	3

	id	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	last_review	reviews_per_month	calculated_host_listings_count
27962	21867710	Amazing room in the centre on Manhatten! Welcome!	159156636	NaN	Manhattan	Hell's Kitchen	40.75668	-73.99097	Private room	120	1	89	2019-01-01	5.16	3
28042	21901156	Best Deal! Lovely place in Manhattan! Time Squ...	159156636	NaN	Manhattan	Hell's Kitchen	40.75656	-73.99063	Private room	120	1	109	2019-01-01	5.97	3
28274	21990808	*SLEEPING IN THE GOLDEN PLACE*	159156636	NaN	Manhattan	Hell's Kitchen	40.75835	-73.99065	Private room	120	1	88	2018-12-14	4.93	3

	id	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	last_review	reviews_per_month	calculated_host_listings_count
27962	21867710	Amazing room in the centre on Manhatten! Welcome!	159156636	NaN	Manhattan	Hell's Kitchen	40.75668	-73.99097	Private room	120	1	89	2019-01-01	5.16	3
28042	21901156	Best Deal! Lovely place in Manhattan! Time Squ...	159156636	NaN	Manhattan	Hell's Kitchen	40.75656	-73.99063	Private room	120	1	109	2019-01-01	5.97	3
28274	21990808	*SLEEPING IN THE GOLDEN PLACE*	159156636	NaN	Manhattan	Hell's Kitchen	40.75835	-73.99065	Private room	120	1	88	2018-12-14	4.93	3

	host_id	host_name	price	minimum_nights	number_of_reviews	calculated_host_listings_count
gender
andy	779	779	779	779	779	779
female	17707	17707	17707	17707	17707	17707
male	16795	16795	16795	16795	16795	16795
mostly_female	1501	1501	1501	1501	1501	1501
mostly_male	1380	1380	1380	1380	1380	1380
unknown	10712	10712	10712	10712	10712	10712

	id	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	last_review	reviews_per_month	calculated_host_listings_count	availability_365
8163	6292866	Modern Quiet Gem Near All	32722063	NaN	Brooklyn	East Flatbush	40.65263	-73.93215	Entire home/apt	85	2	182	2019-06-19	3.59	2	318
8852	6786181	R&S Modern Spacious Hideaway	32722063	NaN	Brooklyn	East Flatbush	40.64345	-73.93643	Entire home/apt	100	2	157	2019-06-19	3.18	2	342

	host_id	host_name	price	minimum_nights	number_of_reviews	calculated_host_listings_count	gender
69	4396	Casey	99	1	106	2	andy
125	6197784	Jo	90	1	9	1	andy
146	143027	Ming	195	4	240	1	andy
161	110506	Myung	79	30	28	1	andy
361	526805	Mi	130	7	98	1	andy

	host_id	price	minimum_nights	number_of_reviews	calculated_host_listings_count	gender
host_id	1	0.00243285	-0.0147733	-0.123354	0.0431339	0.0350373
price	0.00243285	1	0.0418248	-0.0389194	0.00669281	0.00926764
minimum_nights	-0.0147733	0.0418248	1	-0.0693091	0.159549	-0.000330843
number_of_reviews	-0.123354	-0.0389194	-0.0693091	1	-0.0594026	0.0105706
calculated_host_listings_count	0.0431339	0.00669281	0.159549	-0.0594026	1	0.0290068
gender	0.0350373	0.00926764	-0.000330843	0.0105706	0.0290068	1

	host_id	price	minimum_nights	number_of_reviews	calculated_host_listings_count	gender
count	1.817500e+04	18175.000000	18175.000000	18175.000000	18175.000000	18175.0
mean	6.540860e+07	154.427785	6.847153	23.062338	3.597304	1.0
std	7.772192e+07	221.460250	18.239724	43.871153	11.131362	0.0
min	2.787000e+03	0.000000	1.000000	0.000000	1.000000	1.0
25%	7.776920e+06	70.000000	1.000000	1.000000	1.000000	1.0
50%	2.864016e+07	110.000000	3.000000	5.000000	1.000000	1.0
75%	1.033570e+08	180.000000	5.000000	23.000000	2.000000	1.0
max	2.743115e+08	9999.000000	1000.000000	488.000000	103.000000	1.0

	host_id	price	minimum_nights	number_of_reviews	calculated_host_listings_count	gender
count	1.920800e+04	19208.000000	19208.000000	19208.000000	19208.000000	19208.0
mean	6.010510e+07	149.872865	6.861047	22.140775	2.968034	0.0
std	7.355637e+07	266.516898	23.298156	43.288731	10.553940	0.0
min	2.845000e+03	0.000000	1.000000	0.000000	1.000000	0.0
25%	6.802500e+06	69.000000	1.000000	1.000000	1.000000	0.0
50%	2.636173e+07	105.000000	3.000000	5.000000	1.000000	0.0
75%	8.707375e+07	170.000000	5.000000	22.000000	2.000000	0.0
max	2.743213e+08	10000.000000	1250.000000	629.000000	121.000000	0.0

	name	host_id	host_name	neighbourhood_group	neighbourhood	latitude	longitude	room_type	price	minimum_nights	number_of_reviews	reviews_per_month	calculated_host_listings_count	availability_365	price_bin
55	Williamsburg 1 bedroom Apartment	56512	Joanna	Brooklyn	Williamsburg	40.70881	-73.95930	Entire home/apt	150	30	105	0.90	1	30	1
59	Bright Nolita Apt w Doorman/Elevators/Gym	60049	Ted	Manhattan	SoHo	40.72210	-73.99775	Entire home/apt	180	30	21	0.30	1	154	2
63	Comfortable 4-bedroom apt in family house.	63588	Dimitri	Brooklyn	Prospect Heights	40.67811	-73.96428	Entire home/apt	200	30	143	1.33	2	297	2
65	Your Heaven in Hells Kitchen	63924	Mark	Manhattan	Hell's Kitchen	40.75979	-73.99119	Private room	52	30	191	1.65	1	191	1
77	Little King of Queens	70091	Justin	Queens	Woodside	40.75038	-73.90334	Private room	70	30	25	0.22	1	324	1

	count	mean	std	min	25%	50%	75%	max
price	47912.0	132.488813	90.059801	10.0	69.0	102.0	175.0	550.0
label	47912.0	0.000000	0.000000	0.0	0.0	0.0	0.0	0.0

	count	mean	std	min	25%	50%	75%	max
price	43.0	5845.860465	2153.079042	3512.0	4050.0	5000.0	7150.0	10000.0
label	43.0	1.000000	0.000000	1.0	1.0	1.0	1.0	1.0

	count	mean	std	min	25%	50%	75%	max
price	892.0	967.94843	507.985407	555.0	650.0	799.0	1000.0	3210.0
label	892.0	2.00000	0.000000	2.0	2.0	2.0	2.0	2.0

numerical -> id, host_id, latitude, longitude, price, minimum_nights, number_of_reviews,¶

reviews_per_month, calculated_host_listings_count, availability_365¶

categorical -> room_type¶

not_sure -> name, host_name, neighbourhood_group, neighbourhood, last_review¶

Plot longitude and latitude of all hosts¶

let's check if gender of owner has any impact of price and number of listing and number of review¶

let's check realation of name and neighbourhood¶

let's do some on the book eda¶

price¶

room_type¶

minimum_nights¶

availability_365¶

Make dictionary that contain neighbourhood_group and their neighbourhood¶

Plot total_review of all neighbourhood_group (1)¶

Plot count of host in neighbourhood_group (2)¶

Relation between neighbourhood_group and other fileds¶

neighbourhood_group with price (3)¶

plot number (4)¶

ANOVA TEST for mean of price in neighbourhood_group¶

Frequency amount of price¶

plot number (5)¶

bining price -> [10, 153, 393, 633, 10000]¶

count of each neighbourhood_group in each price bins¶

plot number (6)¶

plot number (7)¶

Relationb between minimum_nights and price¶

frequency amount of minimum_nights (7)¶

plot count host in each minimum_nights (8)¶

plot number (9)¶

plot price_bin and minimum_nights (10)¶

ANOVA for mean of minimum_nights in each price_bin¶

count of minimum_nights that are equal 30 for each price_bin¶

Relationb between minimum_nights and location¶

ANOVA for mean of minimum_nights in each neighbourhood_group¶

Trim mean¶

rebace ://¶

Make three base bin for price (low, average, ideal)¶

Make DataFrame for each cluster¶

Frequency of price for cluster_0¶

Frequency of price for cluster_1¶

Frequency of price for cluster_2¶

Make 3 cluster for cluster_0¶

let's explore id¶